<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<title>Reranker Framework (ReFr): reranker::StreamTokenizer Class Reference</title>

<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link href="doxygen.css" rel="stylesheet" type="text/css" />

<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
  $(document).ready(function() { searchBox.OnSelectItem(0); });
</script>

</head>
<body>
<div id="top"><!-- do not remove this div! -->


<div id="titlearea">
<table cellspacing="0" cellpadding="0">
 <tbody>
 <tr style="height: 56px;">
  
  
  <td style="padding-left: 0.5em;">
   <div id="projectname">Reranker Framework (ReFr)
   
   </div>
   <div id="projectbrief">Reranking framework for structure prediction and discriminative language modeling</div>
  </td>
  
  
  
 </tr>
 </tbody>
</table>
</div>

<!-- Generated by Doxygen 1.7.6.1 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
  <div id="navrow1" class="tabs">
    <ul class="tablist">
      <li><a href="index.html"><span>Main&#160;Page</span></a></li>
      <li><a href="namespaces.html"><span>Namespaces</span></a></li>
      <li class="current"><a href="annotated.html"><span>Classes</span></a></li>
      <li><a href="files.html"><span>Files</span></a></li>
      <li>
        <div id="MSearchBox" class="MSearchBoxInactive">
        <span class="left">
          <img id="MSearchSelect" src="search/mag_sel.png"
               onmouseover="return searchBox.OnSearchSelectShow()"
               onmouseout="return searchBox.OnSearchSelectHide()"
               alt=""/>
          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)" 
               onblur="searchBox.OnSearchFieldFocus(false)" 
               onkeyup="searchBox.OnSearchFieldChange(event)"/>
          </span><span class="right">
            <a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
          </span>
        </div>
      </li>
    </ul>
  </div>
  <div id="navrow2" class="tabs2">
    <ul class="tablist">
      <li><a href="annotated.html"><span>Class&#160;List</span></a></li>
      <li><a href="classes.html"><span>Class&#160;Index</span></a></li>
      <li><a href="hierarchy.html"><span>Class&#160;Hierarchy</span></a></li>
      <li><a href="functions.html"><span>Class&#160;Members</span></a></li>
    </ul>
  </div>
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark">&#160;</span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark">&#160;</span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark">&#160;</span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark">&#160;</span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark">&#160;</span>Defines</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>

  <div id="nav-path" class="navpath">
    <ul>
      <li class="navelem"><a class="el" href="namespacereranker.html">reranker</a>      </li>
      <li class="navelem"><a class="el" href="classreranker_1_1_stream_tokenizer.html">StreamTokenizer</a>      </li>
    </ul>
  </div>
</div>
<div class="header">
  <div class="summary">
<a href="#pub-types">Public Types</a> &#124;
<a href="#pub-methods">Public Member Functions</a> &#124;
<a href="#pub-static-methods">Static Public Member Functions</a>  </div>
  <div class="headertitle">
<div class="title">reranker::StreamTokenizer Class Reference</div>  </div>
</div><!--header-->
<div class="contents">
<!-- doxytag: class="reranker::StreamTokenizer" -->
<p>A simple class for tokenizing a stream of tokens for the formally specified language used to construct objects for the Reranker framework.  
 <a href="classreranker_1_1_stream_tokenizer.html#details">More...</a></p>

<p><code>#include &lt;<a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>&gt;</code></p>

<p><a href="classreranker_1_1_stream_tokenizer-members.html">List of all members.</a></p>
<table class="memberdecls">
<tr><td colspan="2"><h2><a name="pub-types"></a>
Public Types</h2></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">enum &#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">TokenType</a> { <br/>
&#160;&#160;<a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa5c9b72ada7604069ee7c91fb7c77cc88">EOF_TYPE</a>, 
<a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daaf92bc177d9920c389a149fa718c9ff3a">RESERVED_CHAR</a>, 
<a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa900de35c73023cf15ec3954852da6809">RESERVED_WORD</a>, 
<a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa6719f11fc4a17c542500088211a08df3">STRING</a>, 
<br/>
&#160;&#160;<a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa033acb728cba240ddf92404a8e709b79">NUMBER</a>, 
<a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412daa6f3f909fb7f4fdc76c355f131a0cd392">IDENTIFIER</a>
<br/>
 }</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">The set of types of tokens read by this stream tokenizer.  <a href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">More...</a><br/></td></tr>
<tr><td colspan="2"><h2><a name="pub-methods"></a>
Public Member Functions</h2></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a9f58e55e682573883cd8872da9dce149">StreamTokenizer</a> (istream &amp;is, const char *reserved_chars=default_reserved_chars)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Constructs a new instance around the specified byte stream.  <a href="#a9f58e55e682573883cd8872da9dce149"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a71d2bdbd46b7225892d86aed90fabed1">StreamTokenizer</a> (const string &amp;s, const char *reserved_chars=default_reserved_chars)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Constructs a new instance around the specified string.  <a href="#a71d2bdbd46b7225892d86aed90fabed1"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">void&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a60647671d07982e301b6d517192cc537">set_reserved_words</a> (set&lt; string &gt; &amp;reserved_words)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Sets the set of &ldquo;reserved words&rdquo; used by this stream tokenizer.  <a href="#a60647671d07982e301b6d517192cc537"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">virtual&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a4e49fac1cca5675cddb0bad5b28db74f">~StreamTokenizer</a> ()</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Destroys this instance.  <a href="#a4e49fac1cca5675cddb0bad5b28db74f"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">string&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#af4765dcb4f25bb60a13f879536d68288">str</a> ()</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns the entire sequence of characters read so far by this stream tokenizer as a newly constructed string object.  <a href="#af4765dcb4f25bb60a13f879536d68288"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">size_t&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a6afa23fd8d78598209e49b8723683b79">tellg</a> () const </td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns the number of bytes read from the underlying byte stream.  <a href="#a6afa23fd8d78598209e49b8723683b79"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">size_t&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a9c45df83d9bd8319465459e8510e741f">line_number</a> () const </td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns the number of lines read from the underlying byte stream, where a line is any number of bytes followed by a newline character (i.e., this is ASCII-centric).  <a href="#a9c45df83d9bd8319465459e8510e741f"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">bool&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a1e015bd9bd6dd38c6c1cd4993e44a77e">HasNext</a> () const </td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns whether there is another token in the token stream.  <a href="#a1e015bd9bd6dd38c6c1cd4993e44a77e"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">bool&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a5838d7201c6a6f5a1131f5cfe4f1a8af">HasPrev</a> () const </td></tr>
<tr><td class="memItemLeft" align="right" valign="top">string&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a5ca21d5cc8edfba7e220e71a553973ce">PeekPrev</a> () const </td></tr>
<tr><td class="memItemLeft" align="right" valign="top">size_t&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#ae11b72f787861e255191a574a9391e43">PeekPrevTokenStart</a> () const </td></tr>
<tr><td class="memItemLeft" align="right" valign="top"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">TokenType</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#ae3d8731cc6ffb815a8a2fe44f9d27bf9">PeekPrevTokenType</a> () const </td></tr>
<tr><td class="memItemLeft" align="right" valign="top">string&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a8fad418b62ae370295fe324229e5d43e">Next</a> ()</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns the next token in the token stream.  <a href="#a8fad418b62ae370295fe324229e5d43e"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">size_t&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a2f79a9d009fac16c76c9bcf653fb88cc">PeekTokenStart</a> () const </td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns the next token&rsquo;s start position.  <a href="#a2f79a9d009fac16c76c9bcf653fb88cc"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">TokenType</a>&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a69295644dad07656c4790d6d5ed87557">PeekTokenType</a> () const </td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns the type of the next token.  <a href="#a69295644dad07656c4790d6d5ed87557"></a><br/></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">string&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a7f53bc3134419cb7b7f00bbd39430b42">Peek</a> () const </td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns the next token that would be returned by the <a class="el" href="classreranker_1_1_stream_tokenizer.html#a8fad418b62ae370295fe324229e5d43e">Next</a> method.  <a href="#a7f53bc3134419cb7b7f00bbd39430b42"></a><br/></td></tr>
<tr><td colspan="2"><h2><a name="pub-static-methods"></a>
Static Public Member Functions</h2></td></tr>
<tr><td class="memItemLeft" align="right" valign="top">static const char *&#160;</td><td class="memItemRight" valign="bottom"><a class="el" href="classreranker_1_1_stream_tokenizer.html#ad9765fd0018581aa96914ac773cbe450">TypeName</a> (<a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">TokenType</a> token_type)</td></tr>
<tr><td class="mdescLeft">&#160;</td><td class="mdescRight">Returns a string type name for the specified TokenType constant.  <a href="#ad9765fd0018581aa96914ac773cbe450"></a><br/></td></tr>
</table>
<hr/><a name="details" id="details"></a><h2>Detailed Description</h2>
<div class="textblock"><p>A simple class for tokenizing a stream of tokens for the formally specified language used to construct objects for the Reranker framework. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00084">84</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>
</div><hr/><h2>Member Enumeration Documentation</h2>
<a class="anchor" id="a43c556f438853bde82330f852de412da"></a><!-- doxytag: member="reranker::StreamTokenizer::TokenType" ref="a43c556f438853bde82330f852de412da" args="" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">enum <a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">reranker::StreamTokenizer::TokenType</a></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>The set of types of tokens read by this stream tokenizer. </p>
<dl class="see"><dt><b>See also:</b></dt><dd><a class="el" href="classreranker_1_1_stream_tokenizer.html#a69295644dad07656c4790d6d5ed87557" title="Returns the type of the next token.">PeekTokenType</a> </dd></dl>
<dl><dt><b>Enumerator: </b></dt><dd><table border="0" cellspacing="2" cellpadding="0">
<tr><td valign="top"><em><a class="anchor" id="a43c556f438853bde82330f852de412daa5c9b72ada7604069ee7c91fb7c77cc88"></a><!-- doxytag: member="EOF_TYPE" ref="a43c556f438853bde82330f852de412daa5c9b72ada7604069ee7c91fb7c77cc88" args="" -->EOF_TYPE</em>&nbsp;</td><td>
</td></tr>
<tr><td valign="top"><em><a class="anchor" id="a43c556f438853bde82330f852de412daaf92bc177d9920c389a149fa718c9ff3a"></a><!-- doxytag: member="RESERVED_CHAR" ref="a43c556f438853bde82330f852de412daaf92bc177d9920c389a149fa718c9ff3a" args="" -->RESERVED_CHAR</em>&nbsp;</td><td>
</td></tr>
<tr><td valign="top"><em><a class="anchor" id="a43c556f438853bde82330f852de412daa900de35c73023cf15ec3954852da6809"></a><!-- doxytag: member="RESERVED_WORD" ref="a43c556f438853bde82330f852de412daa900de35c73023cf15ec3954852da6809" args="" -->RESERVED_WORD</em>&nbsp;</td><td>
</td></tr>
<tr><td valign="top"><em><a class="anchor" id="a43c556f438853bde82330f852de412daa6719f11fc4a17c542500088211a08df3"></a><!-- doxytag: member="STRING" ref="a43c556f438853bde82330f852de412daa6719f11fc4a17c542500088211a08df3" args="" -->STRING</em>&nbsp;</td><td>
</td></tr>
<tr><td valign="top"><em><a class="anchor" id="a43c556f438853bde82330f852de412daa033acb728cba240ddf92404a8e709b79"></a><!-- doxytag: member="NUMBER" ref="a43c556f438853bde82330f852de412daa033acb728cba240ddf92404a8e709b79" args="" -->NUMBER</em>&nbsp;</td><td>
</td></tr>
<tr><td valign="top"><em><a class="anchor" id="a43c556f438853bde82330f852de412daa6f3f909fb7f4fdc76c355f131a0cd392"></a><!-- doxytag: member="IDENTIFIER" ref="a43c556f438853bde82330f852de412daa6f3f909fb7f4fdc76c355f131a0cd392" args="" -->IDENTIFIER</em>&nbsp;</td><td>
</td></tr>
</table>
</dd>
</dl>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00089">89</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<hr/><h2>Constructor &amp; Destructor Documentation</h2>
<a class="anchor" id="a9f58e55e682573883cd8872da9dce149"></a><!-- doxytag: member="reranker::StreamTokenizer::StreamTokenizer" ref="a9f58e55e682573883cd8872da9dce149" args="(istream &amp;is, const char *reserved_chars=default_reserved_chars)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a9f58e55e682573883cd8872da9dce149">reranker::StreamTokenizer::StreamTokenizer</a> </td>
          <td>(</td>
          <td class="paramtype">istream &amp;&#160;</td>
          <td class="paramname"><em>is</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const char *&#160;</td>
          <td class="paramname"><em>reserved_chars</em> = <code>default_reserved_chars</code>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td><code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Constructs a new instance around the specified byte stream. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramname">is</td><td>the input byte stream for this stream tokenizer to use </td></tr>
    <tr><td class="paramname">reserved_chars</td><td>the set of single characters serving as &ldquo;reserved characters&rdquo; </td></tr>
  </table>
  </dd>
</dl>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00112">112</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a71d2bdbd46b7225892d86aed90fabed1"></a><!-- doxytag: member="reranker::StreamTokenizer::StreamTokenizer" ref="a71d2bdbd46b7225892d86aed90fabed1" args="(const string &amp;s, const char *reserved_chars=default_reserved_chars)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a9f58e55e682573883cd8872da9dce149">reranker::StreamTokenizer::StreamTokenizer</a> </td>
          <td>(</td>
          <td class="paramtype">const string &amp;&#160;</td>
          <td class="paramname"><em>s</em>, </td>
        </tr>
        <tr>
          <td class="paramkey"></td>
          <td></td>
          <td class="paramtype">const char *&#160;</td>
          <td class="paramname"><em>reserved_chars</em> = <code>default_reserved_chars</code>&#160;</td>
        </tr>
        <tr>
          <td></td>
          <td>)</td>
          <td></td><td><code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Constructs a new instance around the specified string. </p>
<dl class="params"><dt><b>Parameters:</b></dt><dd>
  <table class="params">
    <tr><td class="paramname">s</td><td>the string providing the stream of characters for this stream tokenizer to use </td></tr>
    <tr><td class="paramname">reserved_chars</td><td>the set of single characters serving as &ldquo;reserved characters&rdquo; </td></tr>
  </table>
  </dd>
</dl>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00125">125</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a4e49fac1cca5675cddb0bad5b28db74f"></a><!-- doxytag: member="reranker::StreamTokenizer::~StreamTokenizer" ref="a4e49fac1cca5675cddb0bad5b28db74f" args="()" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">virtual <a class="el" href="classreranker_1_1_stream_tokenizer.html#a4e49fac1cca5675cddb0bad5b28db74f">reranker::StreamTokenizer::~StreamTokenizer</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td><code> [inline, virtual]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Destroys this instance. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00139">139</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<hr/><h2>Member Function Documentation</h2>
<a class="anchor" id="a1e015bd9bd6dd38c6c1cd4993e44a77e"></a><!-- doxytag: member="reranker::StreamTokenizer::HasNext" ref="a1e015bd9bd6dd38c6c1cd4993e44a77e" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">bool <a class="el" href="classreranker_1_1_stream_tokenizer.html#a1e015bd9bd6dd38c6c1cd4993e44a77e">reranker::StreamTokenizer::HasNext</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns whether there is another token in the token stream. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00157">157</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a5838d7201c6a6f5a1131f5cfe4f1a8af"></a><!-- doxytag: member="reranker::StreamTokenizer::HasPrev" ref="a5838d7201c6a6f5a1131f5cfe4f1a8af" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">bool <a class="el" href="classreranker_1_1_stream_tokenizer.html#a5838d7201c6a6f5a1131f5cfe4f1a8af">reranker::StreamTokenizer::HasPrev</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00159">159</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a9c45df83d9bd8319465459e8510e741f"></a><!-- doxytag: member="reranker::StreamTokenizer::line_number" ref="a9c45df83d9bd8319465459e8510e741f" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">size_t <a class="el" href="classreranker_1_1_stream_tokenizer.html#a9c45df83d9bd8319465459e8510e741f">reranker::StreamTokenizer::line_number</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns the number of lines read from the underlying byte stream, where a line is any number of bytes followed by a newline character (i.e., this is ASCII-centric). </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00154">154</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a8fad418b62ae370295fe324229e5d43e"></a><!-- doxytag: member="reranker::StreamTokenizer::Next" ref="a8fad418b62ae370295fe324229e5d43e" args="()" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">string <a class="el" href="classreranker_1_1_stream_tokenizer.html#a8fad418b62ae370295fe324229e5d43e">reranker::StreamTokenizer::Next</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td><code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns the next token in the token stream. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00172">172</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a7f53bc3134419cb7b7f00bbd39430b42"></a><!-- doxytag: member="reranker::StreamTokenizer::Peek" ref="a7f53bc3134419cb7b7f00bbd39430b42" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">string <a class="el" href="classreranker_1_1_stream_tokenizer.html#a7f53bc3134419cb7b7f00bbd39430b42">reranker::StreamTokenizer::Peek</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns the next token that would be returned by the <a class="el" href="classreranker_1_1_stream_tokenizer.html#a8fad418b62ae370295fe324229e5d43e">Next</a> method. </p>
<p>The return value of this method is only valid when <a class="el" href="classreranker_1_1_stream_tokenizer.html#a1e015bd9bd6dd38c6c1cd4993e44a77e">HasNext</a> returns <code>true</code>. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00201">201</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a5ca21d5cc8edfba7e220e71a553973ce"></a><!-- doxytag: member="reranker::StreamTokenizer::PeekPrev" ref="a5ca21d5cc8edfba7e220e71a553973ce" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">string <a class="el" href="classreranker_1_1_stream_tokenizer.html#a5ca21d5cc8edfba7e220e71a553973ce">reranker::StreamTokenizer::PeekPrev</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00161">161</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="ae11b72f787861e255191a574a9391e43"></a><!-- doxytag: member="reranker::StreamTokenizer::PeekPrevTokenStart" ref="ae11b72f787861e255191a574a9391e43" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">size_t <a class="el" href="classreranker_1_1_stream_tokenizer.html#ae11b72f787861e255191a574a9391e43">reranker::StreamTokenizer::PeekPrevTokenStart</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00163">163</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="ae3d8731cc6ffb815a8a2fe44f9d27bf9"></a><!-- doxytag: member="reranker::StreamTokenizer::PeekPrevTokenType" ref="ae3d8731cc6ffb815a8a2fe44f9d27bf9" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">TokenType</a> <a class="el" href="classreranker_1_1_stream_tokenizer.html#ae3d8731cc6ffb815a8a2fe44f9d27bf9">reranker::StreamTokenizer::PeekPrevTokenType</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00167">167</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a2f79a9d009fac16c76c9bcf653fb88cc"></a><!-- doxytag: member="reranker::StreamTokenizer::PeekTokenStart" ref="a2f79a9d009fac16c76c9bcf653fb88cc" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">size_t <a class="el" href="classreranker_1_1_stream_tokenizer.html#a2f79a9d009fac16c76c9bcf653fb88cc">reranker::StreamTokenizer::PeekTokenStart</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns the next token&rsquo;s start position. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00189">189</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a69295644dad07656c4790d6d5ed87557"></a><!-- doxytag: member="reranker::StreamTokenizer::PeekTokenType" ref="a69295644dad07656c4790d6d5ed87557" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">TokenType</a> <a class="el" href="classreranker_1_1_stream_tokenizer.html#a69295644dad07656c4790d6d5ed87557">reranker::StreamTokenizer::PeekTokenType</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns the type of the next token. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00194">194</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a60647671d07982e301b6d517192cc537"></a><!-- doxytag: member="reranker::StreamTokenizer::set_reserved_words" ref="a60647671d07982e301b6d517192cc537" args="(set&lt; string &gt; &amp;reserved_words)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">void <a class="el" href="classreranker_1_1_stream_tokenizer.html#a60647671d07982e301b6d517192cc537">reranker::StreamTokenizer::set_reserved_words</a> </td>
          <td>(</td>
          <td class="paramtype">set&lt; string &gt; &amp;&#160;</td>
          <td class="paramname"><em>reserved_words</em></td><td>)</td>
          <td><code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Sets the set of &ldquo;reserved words&rdquo; used by this stream tokenizer. </p>
<p>Should be invoked just after construction time. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00134">134</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="af4765dcb4f25bb60a13f879536d68288"></a><!-- doxytag: member="reranker::StreamTokenizer::str" ref="af4765dcb4f25bb60a13f879536d68288" args="()" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">string <a class="el" href="classreranker_1_1_stream_tokenizer.html#af4765dcb4f25bb60a13f879536d68288">reranker::StreamTokenizer::str</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td><code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns the entire sequence of characters read so far by this stream tokenizer as a newly constructed string object. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00145">145</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="a6afa23fd8d78598209e49b8723683b79"></a><!-- doxytag: member="reranker::StreamTokenizer::tellg" ref="a6afa23fd8d78598209e49b8723683b79" args="() const " -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">size_t <a class="el" href="classreranker_1_1_stream_tokenizer.html#a6afa23fd8d78598209e49b8723683b79">reranker::StreamTokenizer::tellg</a> </td>
          <td>(</td>
          <td class="paramname"></td><td>)</td>
          <td> const<code> [inline]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns the number of bytes read from the underlying byte stream. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00149">149</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<a class="anchor" id="ad9765fd0018581aa96914ac773cbe450"></a><!-- doxytag: member="reranker::StreamTokenizer::TypeName" ref="ad9765fd0018581aa96914ac773cbe450" args="(TokenType token_type)" -->
<div class="memitem">
<div class="memproto">
      <table class="memname">
        <tr>
          <td class="memname">static const char* <a class="el" href="classreranker_1_1_stream_tokenizer.html#ad9765fd0018581aa96914ac773cbe450">reranker::StreamTokenizer::TypeName</a> </td>
          <td>(</td>
          <td class="paramtype"><a class="el" href="classreranker_1_1_stream_tokenizer.html#a43c556f438853bde82330f852de412da">TokenType</a>&#160;</td>
          <td class="paramname"><em>token_type</em></td><td>)</td>
          <td><code> [inline, static]</code></td>
        </tr>
      </table>
</div>
<div class="memdoc">

<p>Returns a string type name for the specified TokenType constant. </p>

<p>Definition at line <a class="el" href="stream-tokenizer_8_h_source.html#l00099">99</a> of file <a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a>.</p>

</div>
</div>
<hr/>The documentation for this class was generated from the following files:<ul>
<li><a class="el" href="stream-tokenizer_8_h_source.html">stream-tokenizer.H</a></li>
<li><a class="el" href="stream-tokenizer_8_c_source.html">stream-tokenizer.C</a></li>
</ul>
</div><!-- contents -->
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
     onmouseover="return searchBox.OnSearchSelectShow()"
     onmouseout="return searchBox.OnSearchSelectHide()"
     onkeydown="return searchBox.OnSearchSelectKey(event)">
<a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(0)"><span class="SelectionMark">&#160;</span>All</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(1)"><span class="SelectionMark">&#160;</span>Classes</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(2)"><span class="SelectionMark">&#160;</span>Namespaces</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(3)"><span class="SelectionMark">&#160;</span>Files</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(4)"><span class="SelectionMark">&#160;</span>Functions</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(5)"><span class="SelectionMark">&#160;</span>Variables</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(6)"><span class="SelectionMark">&#160;</span>Typedefs</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(7)"><span class="SelectionMark">&#160;</span>Enumerations</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(8)"><span class="SelectionMark">&#160;</span>Enumerator</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(9)"><span class="SelectionMark">&#160;</span>Friends</a><a class="SelectItem" href="javascript:void(0)" onclick="searchBox.OnSelectItem(10)"><span class="SelectionMark">&#160;</span>Defines</a></div>

<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0" 
        name="MSearchResults" id="MSearchResults">
</iframe>
</div>



<hr class="footer"/><address class="footer"><small>
Generated on Tue Apr 9 2013 11:56:32 for Reranker Framework (ReFr) by &#160;<a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.7.6.1
</small></address>

</body>
</html>
